{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "f168fcee",
   "metadata": {},
   "source": [
    "[Python爬取二手房源数据，可视化分析二手房市场行情数据 - 知乎](https://zhuanlan.zhihu.com/p/415367897)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "3244b1c6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://pyecharts.org/#/zh-cn/quickstart?id=%e5%a6%82%e4%bd%95%e5%ae%89%e8%a3%85\n",
    "# pip install pyecharts\n",
    "import pandas as pd\n",
    "from pyecharts.charts import Map\n",
    "from pyecharts.charts import Bar\n",
    "from pyecharts.charts import Line\n",
    "from pyecharts.charts import Grid\n",
    "from pyecharts.charts import Pie\n",
    "from pyecharts.charts import Scatter\n",
    "from pyecharts import options as opts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "365578ee",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>标题</th>\n",
       "      <th>市区</th>\n",
       "      <th>小区</th>\n",
       "      <th>户型</th>\n",
       "      <th>朝向</th>\n",
       "      <th>楼层</th>\n",
       "      <th>装修情况</th>\n",
       "      <th>电梯</th>\n",
       "      <th>面积(㎡)</th>\n",
       "      <th>价格(万元)</th>\n",
       "      <th>年份</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>世纪星城带电梯顶层复式 楼上没有算房本面积</td>\n",
       "      <td>通州</td>\n",
       "      <td>世纪星城</td>\n",
       "      <td>4室2厅</td>\n",
       "      <td>南 北</td>\n",
       "      <td>10</td>\n",
       "      <td>简装</td>\n",
       "      <td>有</td>\n",
       "      <td>92.20</td>\n",
       "      <td>550</td>\n",
       "      <td>2005</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>雅丽世居 2室2厅 南 北</td>\n",
       "      <td>通州</td>\n",
       "      <td>雅丽世居</td>\n",
       "      <td>2室2厅</td>\n",
       "      <td>南 北</td>\n",
       "      <td>18</td>\n",
       "      <td>精装</td>\n",
       "      <td>有</td>\n",
       "      <td>103.44</td>\n",
       "      <td>360</td>\n",
       "      <td>2005</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>东区全南向，满五唯一，高层不临街，采光好、视野佳</td>\n",
       "      <td>朝阳</td>\n",
       "      <td>华纺易城</td>\n",
       "      <td>2室1厅</td>\n",
       "      <td>南</td>\n",
       "      <td>18</td>\n",
       "      <td>精装</td>\n",
       "      <td>有</td>\n",
       "      <td>100.41</td>\n",
       "      <td>865</td>\n",
       "      <td>2005</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>知春路49号，东南向三居室，带电梯精装</td>\n",
       "      <td>海淀</td>\n",
       "      <td>希格玛公寓</td>\n",
       "      <td>3室2厅</td>\n",
       "      <td>东南</td>\n",
       "      <td>20</td>\n",
       "      <td>精装</td>\n",
       "      <td>有</td>\n",
       "      <td>161.37</td>\n",
       "      <td>1360</td>\n",
       "      <td>未知</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>海特花园东区 3室1厅 南 北</td>\n",
       "      <td>石景山</td>\n",
       "      <td>海特花园东区</td>\n",
       "      <td>3室1厅</td>\n",
       "      <td>南 北</td>\n",
       "      <td>7</td>\n",
       "      <td>简装</td>\n",
       "      <td>无</td>\n",
       "      <td>87.33</td>\n",
       "      <td>439</td>\n",
       "      <td>2001</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                         标题   市区      小区    户型   朝向  楼层 装修情况 电梯   面积(㎡)  \\\n",
       "0     世纪星城带电梯顶层复式 楼上没有算房本面积   通州    世纪星城  4室2厅  南 北  10   简装  有   92.20   \n",
       "1             雅丽世居 2室2厅 南 北   通州    雅丽世居  2室2厅  南 北  18   精装  有  103.44   \n",
       "2  东区全南向，满五唯一，高层不临街，采光好、视野佳   朝阳    华纺易城  2室1厅    南  18   精装  有  100.41   \n",
       "3       知春路49号，东南向三居室，带电梯精装   海淀   希格玛公寓  3室2厅   东南  20   精装  有  161.37   \n",
       "4           海特花园东区 3室1厅 南 北  石景山  海特花园东区  3室1厅  南 北   7   简装  无   87.33   \n",
       "\n",
       "   价格(万元)    年份  \n",
       "0     550  2005  \n",
       "1     360  2005  \n",
       "2     865  2005  \n",
       "3    1360    未知  \n",
       "4     439  2001  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('二手房数据.csv', encoding = 'utf-8')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "8c552161",
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'count' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m/var/folders/mj/gzzjyp5s3j5fq8jywxpy4dyc0000gn/T/ipykernel_46262/1384109526.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      5\u001b[0m m = (\n\u001b[1;32m      6\u001b[0m         \u001b[0mMap\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m         \u001b[0;34m.\u001b[0m\u001b[0madd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m''\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mz\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mz\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnew\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcount\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'北京'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      8\u001b[0m         .set_global_opts(\n\u001b[1;32m      9\u001b[0m             \u001b[0mtitle_opts\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mopts\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTitleOpts\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtitle\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'北京市二手房各区分布'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'count' is not defined"
     ]
    }
   ],
   "source": [
    "# 各城区二手房数量北京市地图\n",
    "region=['通州','朝阳','海淀','石景山','丰台','昌平','西城','大兴','顺义','亦庄开发']\n",
    "count=df['']\n",
    "new = [x + '区' for x in region]\n",
    "m = (\n",
    "        Map()\n",
    "        .add('', [list(z) for z in zip(new, count)], '北京')\n",
    "        .set_global_opts(\n",
    "            title_opts=opts.TitleOpts(title='北京市二手房各区分布'),\n",
    "            visualmap_opts=opts.VisualMapOpts(max_=3000),\n",
    "        )\n",
    "    )\n",
    "m.render_notebook()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "57fadecc",
   "metadata": {},
   "outputs": [],
   "source": [
    "my_text=''\n",
    "# 解析数据\n",
    "selector_1 = parsel.Selector(response.text)\n",
    "# 把获取到response.text 数据内容转成 selector 对象\n",
    "href = selector_1.css('div.leftContent li div.title a::attr(href)').getall()\n",
    "for link in href:\n",
    "    html_data = requests.get(url=link, headers=headers).text\n",
    "    selector = parsel.Selector(html_data)\n",
    "    # css选择器 语法\n",
    "    # try:\n",
    "    title = selector.css('.title h1::text').get() # 标题\n",
    "    area = selector.css('.areaName .info a:nth-child(1)::text').get()  # 区域\n",
    "    community_name = selector.css('.communityName .info::text').get()  # 小区\n",
    "    room = selector.css('.room .mainInfo::text').get()  # 户型\n",
    "    room_type = selector.css('.type .mainInfo::text').get()  # 朝向\n",
    "    height = selector.css('.room .subInfo::text').get().split('/')[-1]  # 楼层\n",
    "    # 中楼层/共5层 split('/') 进行字符串分割  ['中楼层', '共5层'] [-1]\n",
    "    # ['中楼层', '共5层'][-1] 列表索引位置取值 取列表中最后一个元素  共5层\n",
    "    # re.findall('共(\\d+)层', 共5层) >>>  [5][0] >>> 5\n",
    "    height = re.findall('共(\\d+)层', height)[0]\n",
    "    sub_info = selector.css('.type .subInfo::text').get().split('/')[-1]  # 装修\n",
    "    Elevator = selector.css('.content li:nth-child(12)::text').get()  # 电梯\n",
    "    # if Elevator == '暂无数据电梯' or Elevator == None:\n",
    "    #     Elevator = '无电梯'\n",
    "    house_area = selector.css('.content li:nth-child(3)::text').get().replace('㎡', '')  # 面积\n",
    "    price = selector.css('.price .total::text').get()  # 价格(万元)\n",
    "    date = selector.css('.area .subInfo::text').get().replace('年建', '')  # 年份\n",
    "    dit = {\n",
    "        '标题': title,\n",
    "        '市区': area,\n",
    "        '小区': community_name,\n",
    "        '户型': room,\n",
    "        '朝向': room_type,\n",
    "        '楼层': height,\n",
    "        '装修情况': sub_info,\n",
    "        '电梯': Elevator,\n",
    "        '面积(㎡)': house_area,\n",
    "        '价格(万元)': price,\n",
    "        '年份': date,\n",
    "    }\n",
    "    csv_writer.writerow(dit)\n",
    "    my_text=my_text+'\\t'.join(\n",
    "        [title, area, community_name, room, room_type, height, sub_info, Elevator, house_area, price, date]\n",
    "    )+'\\n'\n",
    "#     print(title, area, community_name, room, room_type, height, sub_info, Elevator, house_area, price, date,\n",
    "#           sep='|')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "e4276996",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "43"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "csv_writer.writeheader()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "aff23301",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('/Users/msxr/develop/tmp/二手房数据.txt', mode='w', encoding='utf-8') as fw:\n",
    "    fw.write(my_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3674ce42",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "python",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
